Imports

import glob
import cv2
import os

import numpy as np 
import pandas as pd 
import tensorflow as tf
import matplotlib.pyplot as plt
import seaborn as sns


from collections import Counter
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import f1_score, roc_auc_score, cohen_kappa_score, precision_score, recall_score, accuracy_score, confusion_matrix
from tensorflow.keras.utils import to_categorical

%matplotlib inline

print(os.listdir("../input/intel-mobileodt-cervical-cancer-screening"))

['additional_Type_2_v2', 'test_stg2.7z', 'fixed_labels_v2.csv', 'train', 'sample_submission.csv', 'test', 'additional_Type_1_v2', 'sample_submission_stg2.csv', 'test_stg2_7z_password.txt', 'additional_Type_3_v2', 'removed_files.csv', 'solution_stg1_release.csv']

Data import

#getting the total number of images in the training set

base_dir = '../input/intel-mobileodt-cervical-cancer-screening'

train_dir = os.path.join(base_dir,'train', 'train')

type1_dir = os.path.join(base_dir,'Type_1')
type2_dir = os.path.join(base_dir,'Type_2')
type3_dir = os.path.join(base_dir,'Type_3')

type1_files = glob.glob(type1_dir+'/*.jpg')
type2_files = glob.glob(type2_dir+'/*.jpg')
type3_files = glob.glob(type3_dir+'/*.jpg')

added_type1_files  =  glob.glob(os.path.join(base_dir, "additional_Type_1_v2", "Type_1")+'/*.jpg')
added_type2_files  =  glob.glob(os.path.join(base_dir, "additional_Type_2_v2", "Type_2")+'/*.jpg')
added_type3_files  =  glob.glob(os.path.join(base_dir, "additional_Type_3_v2", "Type_3")+'/*.jpg')

type1_files = type1_files + added_type1_files
type2_files = type2_files + added_type2_files
type3_files = type3_files + added_type3_files


print('Number of images in a train set of type 1: ', len(type1_files))
print('Number of images in a train set of type 2: ', len(type2_files))
print('Number of images in a train set of type 3: ', len(type3_files))
print('Total number of images in a train set: ', sum([len(type1_files), len(type2_files), len(type3_files)]))

Number of images in a train set of type 1:  1191
Number of images in a train set of type 2:  3567
Number of images in a train set of type 3:  1976
Total number of images in a train set:  6734

# Building a dataframe mapping images and Cancer type

files_df = pd.DataFrame({
    'filename': type1_files + type2_files + type3_files,
    'label': ['Type_1'] * len(type1_files) + ['Type_2'] * len(type2_files) + ['Type_3'] * len(type3_files)
})

files_df

	filename	label
0	../input/intel-mobileodt-cervical-cancer-scree...	Type_1
1	../input/intel-mobileodt-cervical-cancer-scree...	Type_1
2	../input/intel-mobileodt-cervical-cancer-scree...	Type_1
3	../input/intel-mobileodt-cervical-cancer-scree...	Type_1
4	../input/intel-mobileodt-cervical-cancer-scree...	Type_1
...	...	...
6729	../input/intel-mobileodt-cervical-cancer-scree...	Type_3
6730	../input/intel-mobileodt-cervical-cancer-scree...	Type_3
6731	../input/intel-mobileodt-cervical-cancer-scree...	Type_3
6732	../input/intel-mobileodt-cervical-cancer-scree...	Type_3
6733	../input/intel-mobileodt-cervical-cancer-scree...	Type_3

6734 rows × 2 columns

#Shuffle data

random_state = 42

files_df = files_df.sample(frac=1, random_state=random_state)
# files_df = files_df.sample(n=100, random_state=random_state)

files_df

	filename	label
381	../input/intel-mobileodt-cervical-cancer-scree...	Type_1
4696	../input/intel-mobileodt-cervical-cancer-scree...	Type_2
5577	../input/intel-mobileodt-cervical-cancer-scree...	Type_3
2018	../input/intel-mobileodt-cervical-cancer-scree...	Type_2
2185	../input/intel-mobileodt-cervical-cancer-scree...	Type_2
...	...	...
4863	../input/intel-mobileodt-cervical-cancer-scree...	Type_3
1446	../input/intel-mobileodt-cervical-cancer-scree...	Type_2
1188	../input/intel-mobileodt-cervical-cancer-scree...	Type_1
6471	../input/intel-mobileodt-cervical-cancer-scree...	Type_3
2153	../input/intel-mobileodt-cervical-cancer-scree...	Type_2

100 rows × 2 columns

Data exploration

files_df.describe()

	filename	label
count	100	100
unique	100	3
top	../input/intel-mobileodt-cervical-cancer-scree...	Type_2
freq	1	52

#Check for duplicates
len(files_df[files_df.duplicated()])

0

#Get count of each type 
type_count = pd.DataFrame(files_df['label'].value_counts())
type_count

	count
label
Type_2	52
Type_3	34
Type_1	14

print(list(type_count.columns)[0])

count

# Display barplot of type count

plt.figure(figsize = (15, 6))
sns.barplot(x= type_count[list(type_count.columns)[0]], y= type_count.index.to_list())
plt.title('Cervical Cancer Type Distribution')
plt.grid(True)
plt.show()

# Display sample images of types
for label in ('Type_1', 'Type_2', 'Type_3'):
    filepaths = files_df[files_df['label']==label]['filename'].values[:5]
    fig = plt.figure(figsize= (15, 6))
    for i, path in enumerate(filepaths):
        img = cv2.imread(path)
        img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
        img = cv2.resize(img, (224, 224))
        fig.add_subplot(1, 5, i+1)
        plt.imshow(img)
        plt.subplots_adjust(hspace=0.5)
        plt.axis(False)
        plt.title(label)

Data propocessing

# Split training,val and test set : 70:15:15

train_files, test_files, train_labels, test_labels = train_test_split(files_df['filename'].values,
                                                                      files_df['label'].values, 
                                                                      test_size=0.3, 
                                                                      random_state=random_state)

test_files, val_files, test_labels, val_labels = train_test_split(test_files,
                                                                  test_labels, 
                                                                  test_size=0.5, 
                                                                  random_state=random_state)


print('Number of images in train set: ', train_files.shape)
print('Number of images in validation set: ', val_files.shape)
print('Number of images in test set: ', test_files.shape, '\n')

print('Train:', Counter(train_labels), '\nVal:', Counter(val_labels), '\nTest:', Counter(test_labels))

Number of images in train set:  (70,)
Number of images in validation set:  (15,)
Number of images in test set:  (15,) 

Train: Counter({'Type_2': 30, 'Type_3': 27, 'Type_1': 13}) 
Val: Counter({'Type_2': 12, 'Type_3': 2, 'Type_1': 1}) 
Test: Counter({'Type_2': 10, 'Type_3': 5})

def load_images(files, labels):
    features = []
    correct_labels = []
    bad_images = 0
    
    for i in range(len(files)):
        try:
            img = cv2.imread(files[i])
            resized_img = cv2.resize(img, (160, 160))
            
            features.append(np.array(resized_img))
            correct_labels.append(labels[i])
                   
        except Exception as e:
            bad_images+=1
            print('Encoutered bad image')
    print('Bad images ecountered:', bad_images)
    return np.array(features), np.array(correct_labels)

# Load training and evaluation data
train_features, train_labels = load_images(train_files, train_labels)
print('Train images loaded')

val_features, val_labels = load_images(val_files, val_labels)
print('Validation images loaded')

test_features, test_labels = load_images(test_files, test_labels)
print('test images loaded')

Bad images ecountered: 0
Train images loaded
Bad images ecountered: 0
Validation images loaded
Bad images ecountered: 0
test images loaded

# check lengths of training and evaluation  sets
len(train_features), len(train_labels), len(val_features), len(val_labels), len(test_features), len(test_labels) 

(70, 70, 15, 15, 15, 15)

BATCH_SIZE = 32
NUM_CLASSES = 3
EPOCHS = 10
INPUT_SHAPE = (160, 160, 3)

# encode train+val sets text categories with labels
le = LabelEncoder()
le.fit(train_labels)

train_labels_enc = le.transform(train_labels)
val_labels_enc = le.transform(val_labels)

train_labels_1hotenc = tf.keras.utils.to_categorical(train_labels_enc, num_classes=NUM_CLASSES)
val_labels_1hotenc = tf.keras.utils.to_categorical(val_labels_enc, num_classes=NUM_CLASSES)

print(train_labels[:6], train_labels_enc[:6])
print(train_labels[:6], train_labels_1hotenc[:6])

['Type_3' 'Type_2' 'Type_2' 'Type_2' 'Type_2' 'Type_2'] [2 1 1 1 1 1]
['Type_3' 'Type_2' 'Type_2' 'Type_2' 'Type_2' 'Type_2'] [[0. 0. 1.]
 [0. 1. 0.]
 [0. 1. 0.]
 [0. 1. 0.]
 [0. 1. 0.]
 [0. 1. 0.]]

le = LabelEncoder()
le.fit(test_labels)

test_labels_enc = le.transform(test_labels)

test_labels_1hotenc = tf.keras.utils.to_categorical(test_labels_enc, num_classes=NUM_CLASSES)


print(test_labels[:6], test_labels_enc[:6])
print(test_labels[:6], test_labels_1hotenc[:6])

['Type_3' 'Type_2' 'Type_2' 'Type_2' 'Type_3' 'Type_2'] [1 0 0 0 1 0]
['Type_3' 'Type_2' 'Type_2' 'Type_2' 'Type_3' 'Type_2'] [[0. 1. 0.]
 [1. 0. 0.]
 [1. 0. 0.]
 [1. 0. 0.]
 [0. 1. 0.]
 [1. 0. 0.]]

Data augmentation

data_augmentation = tf.keras.Sequential([
  tf.keras.layers.RandomFlip('horizontal'),
  tf.keras.layers.RandomRotation(0.2),
])

plt.figure(figsize=(10, 10))
first_image = train_features[0]
for i in range(9):
    ax = plt.subplot(3, 3, i + 1)
    augmented_image = data_augmentation(tf.expand_dims(first_image, 0))
    plt.imshow(augmented_image[0] / 255)
    plt.axis('off')
        

Metrics

def get_accuracy_metrics(model, train_features=train_features, train_labels=train_labels_enc, test_features=test_features, test_labels=test_labels_enc, val_features=val_features, val_labels=val_labels_enc):    
    train_predicted = np.argmax(model.predict(train_features),axis=1)
    test_predicted = np.argmax(model.predict(test_features),axis=1)
    val_predicted = np.argmax(model.predict(val_features),axis=1)

    print("Train accuracy Score------------>")
    print ("{0:.3f}".format(accuracy_score(train_labels, train_predicted) *100), "%")
    
    print("Val accuracy Score--------->")
    print("{0:.3f}".format(accuracy_score(val_labels, val_predicted)*100), "%")
    
    print("Test accuracy Score--------->")
    print("{0:.3f}".format(accuracy_score(test_labels, test_predicted)*100), "%")
    
    print("F1 Score--------------->")
    print("{0:.3f}".format(f1_score(test_labels, test_predicted, average = 'weighted')*100), "%")
    
    print("Cohen Kappa Score------------->")
    print("{0:.3f}".format(cohen_kappa_score(test_labels, test_predicted)*100), "%")
    
    
    print("ROC AUC Score------------->")
    print("{0:.3f}".format(roc_auc_score(to_categorical(test_labels, num_classes = 3), test_predicted.reshape(-1, 1), multi_class='ovr')*100), "%")
    
    print("Recall-------------->")
    print("{0:.3f}".format(recall_score(test_labels, test_predicted, average = 'weighted')*100), "%")
    
    print("Precision-------------->")
    print("{0:.3f}".format(precision_score(test_labels, test_predicted, average = 'weighted')*100), "%")
    
    cf_matrix_test = confusion_matrix(test_labels, test_predicted)
    cf_matrix_val = confusion_matrix(val_labels, val_predicted)
    
    plt.figure(figsize = (12, 6))
    plt.subplot(121)
    sns.heatmap(cf_matrix_val, annot=True, cmap='Blues')
    plt.title("Val Confusion matrix")
    
    plt.subplot(122)
    sns.heatmap(cf_matrix_test, annot=True, cmap='Blues')
    plt.title("Test Confusion matrix")
    
    plt.show()

General Model Fit

def learning_performance_chart(title, history):
    #plots a chart showing the change in accuracy and loss function over epochs
    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))
    t = f.suptitle(title, fontsize=12)
    f.subplots_adjust(top=0.85, wspace=0.3)

    max_epoch = len(history.history['accuracy'])+1
    epoch_list = list(range(1,max_epoch))
    ax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')
    ax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')
    ax1.set_xticks(np.arange(1, max_epoch, 5))
    ax1.set_ylabel('Accuracy Value')
    ax1.set_xlabel('Epoch')
    ax1.set_title('Accuracy')
    l1 = ax1.legend(loc="best")

    ax2.plot(epoch_list, history.history['loss'], label='Train Loss')
    ax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')
    ax2.set_xticks(np.arange(1, max_epoch, 5))
    ax2.set_ylabel('Loss Value')
    ax2.set_xlabel('Epoch')
    ax2.set_title('Loss')
    l2 = ax2.legend(loc="best")

    

def fit_model(model_name, base_model, train_features, train_labels, validate_it,training = False, epochs = EPOCHS, batch_size= BATCH_SIZE):
    
    inputs = tf.keras.Input(shape=INPUT_SHAPE)
    
    x = data_augmentation(inputs)
    x = base_model(x, training=training)
    
    if not model_name.startswith('CNN'):
        x = tf.keras.layers.GlobalAveragePooling2D()(x)
    
    x = tf.keras.layers.Dropout(0.2)(x)
    
    outputs = tf.keras.layers.Dense(3, activation='softmax')(x)
    
    model = tf.keras.Model(inputs, outputs)
    
    es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=5)
    
    model.compile(loss='categorical_crossentropy', optimizer ='adam', metrics=['accuracy'])
    
    print("Model Summary.")
    
    print(model.summary())
    
    history = model.fit(x=train_features,y=train_labels ,validation_data=validate_it, epochs=epochs, batch_size=batch_size, verbose=1, callbacks=[es])

    learning_performance_chart(title="{} learning performance.".format(model_name), history=history)
    
    return model

CNN2

model = tf.keras.Sequential([
 tf.keras.layers.BatchNormalization(),
 tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
 tf.keras.layers.BatchNormalization(),
 tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
 tf.keras.layers.Flatten(),
])

cnn2 = fit_model("CNN2", model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc), training=True)

Model Summary.
Model: "model_8"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_11 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 sequential_9 (Sequential)   (None, 3276800)           75916     
                                                                 
 dropout_15 (Dropout)        (None, 3276800)           0         
                                                                 
 dense_8 (Dense)             (None, 3)                 9830403   
                                                                 
=================================================================
Total params: 9,906,319
Trainable params: 9,906,185
Non-trainable params: 134
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 3s 904ms/step - loss: 231.7615 - accuracy: 0.4857 - val_loss: 108.2145 - val_accuracy: 0.8000
Epoch 2/10
3/3 [==============================] - 3s 880ms/step - loss: 178.6072 - accuracy: 0.4571 - val_loss: 703.4716 - val_accuracy: 0.0667
Epoch 3/10
3/3 [==============================] - 3s 930ms/step - loss: 362.9744 - accuracy: 0.1714 - val_loss: 79.9927 - val_accuracy: 0.4000
Epoch 4/10
3/3 [==============================] - 3s 892ms/step - loss: 95.4329 - accuracy: 0.5429 - val_loss: 243.1655 - val_accuracy: 0.2667
Epoch 5/10
3/3 [==============================] - 3s 785ms/step - loss: 165.0757 - accuracy: 0.4000 - val_loss: 482.3639 - val_accuracy: 0.2000
Epoch 6/10
3/3 [==============================] - 3s 762ms/step - loss: 202.6850 - accuracy: 0.5143 - val_loss: 107.9315 - val_accuracy: 0.8000
Epoch 7/10
3/3 [==============================] - 3s 808ms/step - loss: 397.2995 - accuracy: 0.4286 - val_loss: 61.4279 - val_accuracy: 0.5333
Epoch 8/10
3/3 [==============================] - 3s 720ms/step - loss: 179.2743 - accuracy: 0.3286 - val_loss: 338.7691 - val_accuracy: 0.2000
Epoch 9/10
3/3 [==============================] - 3s 791ms/step - loss: 135.3957 - accuracy: 0.4429 - val_loss: 36.5428 - val_accuracy: 0.5333
Epoch 10/10
3/3 [==============================] - 3s 876ms/step - loss: 70.9520 - accuracy: 0.5857 - val_loss: 140.9207 - val_accuracy: 0.2667

print('CNN2 performance on the test set:')
get_accuracy_metrics(cnn2)

CNN2 performance on the test set:
3/3 [==============================] - 1s 195ms/step
1/1 [==============================] - 0s 169ms/step
1/1 [==============================] - 0s 169ms/step
Train accuracy Score------------>
41.429 %
Val accuracy Score--------->
26.667 %
Test accuracy Score--------->
53.333 %
F1 Score--------------->
51.852 %
Cohen Kappa Score------------->
-10.526 %
ROC AUC Score------------->
55.000 %
Recall-------------->
53.333 %
Precision-------------->
50.758 %

CNN3

model = tf.keras.Sequential([
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Flatten(),
    ])

cnn3 = fit_model("CNN3", model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc), training=True)

Model Summary.
Model: "model_9"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_12 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 sequential_10 (Sequential)  (None, 819200)            223500    
                                                                 
 dropout_17 (Dropout)        (None, 819200)            0         
                                                                 
 dense_9 (Dense)             (None, 3)                 2457603   
                                                                 
=================================================================
Total params: 2,681,103
Trainable params: 2,680,969
Non-trainable params: 134
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 4s 990ms/step - loss: 78.1532 - accuracy: 0.2286 - val_loss: 7.8706 - val_accuracy: 0.8000
Epoch 2/10
3/3 [==============================] - 3s 864ms/step - loss: 14.7362 - accuracy: 0.4286 - val_loss: 0.9287 - val_accuracy: 0.7333
Epoch 3/10
3/3 [==============================] - 3s 838ms/step - loss: 1.0454 - accuracy: 0.5143 - val_loss: 1.1029 - val_accuracy: 0.2000
Epoch 4/10
3/3 [==============================] - 3s 852ms/step - loss: 1.0944 - accuracy: 0.4857 - val_loss: 1.1000 - val_accuracy: 0.1333
Epoch 5/10
3/3 [==============================] - 3s 829ms/step - loss: 1.0970 - accuracy: 0.3857 - val_loss: 1.0989 - val_accuracy: 0.1333
Epoch 6/10
3/3 [==============================] - 3s 843ms/step - loss: 1.0978 - accuracy: 0.3857 - val_loss: 1.0992 - val_accuracy: 0.1333
Epoch 7/10
3/3 [==============================] - 3s 913ms/step - loss: 1.0971 - accuracy: 0.4000 - val_loss: 1.0988 - val_accuracy: 0.2000
Epoch 7: early stopping

print('CNN3 performance on the test set:')
get_accuracy_metrics(cnn3)

CNN3 performance on the test set:
3/3 [==============================] - 1s 308ms/step
1/1 [==============================] - 0s 261ms/step
1/1 [==============================] - 0s 260ms/step
Train accuracy Score------------>
37.143 %
Val accuracy Score--------->
13.333 %
Test accuracy Score--------->
0.000 %
F1 Score--------------->
0.000 %
Cohen Kappa Score------------->
0.000 %
ROC AUC Score------------->
50.000 %
Recall-------------->
0.000 %
Precision-------------->
0.000 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

CNN4

model = tf.keras.Sequential([
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
tf.keras.layers.MaxPooling2D(),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
tf.keras.layers.MaxPooling2D(),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
tf.keras.layers.Flatten(),
])

cnn4 = fit_model("CNN4", model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc), training=True)

Model Summary.
Model: "model_10"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_13 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 sequential_11 (Sequential)  (None, 102400)            297292    
                                                                 
 dropout_20 (Dropout)        (None, 102400)            0         
                                                                 
 dense_10 (Dense)            (None, 3)                 307203    
                                                                 
=================================================================
Total params: 604,495
Trainable params: 604,361
Non-trainable params: 134
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 4s 1s/step - loss: 10.6178 - accuracy: 0.4286 - val_loss: 1.0076 - val_accuracy: 0.6667
Epoch 2/10
3/3 [==============================] - 3s 962ms/step - loss: 1.2460 - accuracy: 0.4143 - val_loss: 2.4352 - val_accuracy: 0.1333
Epoch 3/10
3/3 [==============================] - 3s 893ms/step - loss: 1.1959 - accuracy: 0.4286 - val_loss: 0.8493 - val_accuracy: 0.8000
Epoch 4/10
3/3 [==============================] - 3s 868ms/step - loss: 1.0862 - accuracy: 0.4571 - val_loss: 0.6775 - val_accuracy: 0.8000
Epoch 5/10
3/3 [==============================] - 3s 890ms/step - loss: 1.1260 - accuracy: 0.4286 - val_loss: 0.9135 - val_accuracy: 0.6000
Epoch 6/10
3/3 [==============================] - 3s 905ms/step - loss: 1.0506 - accuracy: 0.4286 - val_loss: 1.0231 - val_accuracy: 0.2000
Epoch 7/10
3/3 [==============================] - 3s 915ms/step - loss: 1.0450 - accuracy: 0.4571 - val_loss: 1.0979 - val_accuracy: 0.2667
Epoch 8/10
3/3 [==============================] - 3s 930ms/step - loss: 1.0445 - accuracy: 0.5143 - val_loss: 1.0180 - val_accuracy: 0.5333
Epoch 9/10
3/3 [==============================] - 3s 916ms/step - loss: 0.9811 - accuracy: 0.5571 - val_loss: 0.7202 - val_accuracy: 0.6667
Epoch 9: early stopping

print('CNN4 performance on the test set:')
get_accuracy_metrics(cnn4)

CNN4 performance on the test set:
3/3 [==============================] - 1s 269ms/step
1/1 [==============================] - 0s 219ms/step
1/1 [==============================] - 0s 273ms/step
Train accuracy Score------------>
47.143 %
Val accuracy Score--------->
66.667 %
Test accuracy Score--------->
33.333 %
F1 Score--------------->
18.519 %
Cohen Kappa Score------------->
6.250 %
ROC AUC Score------------->
60.000 %
Recall-------------->
33.333 %
Precision-------------->
12.821 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

CNN5

model = tf.keras.Sequential([
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters=64, kernel_size=(
        3, 3), padding='same', activation='relu'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters=128, kernel_size=(
        3, 3), padding='same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters=128, kernel_size=(3, 3), padding='same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters=64, kernel_size=(
        3, 3), padding='same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters=32, kernel_size=(
        3, 3), padding='same', activation='relu'),
    tf.keras.layers.Flatten(),
])


cnn5 = fit_model("CNN5", model, train_features, train_labels_1hotenc,
                       (val_features, val_labels_1hotenc), training=True)

Model Summary.
Model: "model_11"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_14 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 sequential_12 (Sequential)  (None, 12800)             315756    
                                                                 
 dropout_24 (Dropout)        (None, 12800)             0         
                                                                 
 dense_11 (Dense)            (None, 3)                 38403     
                                                                 
=================================================================
Total params: 354,159
Trainable params: 354,025
Non-trainable params: 134
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 5s 988ms/step - loss: 2.8182 - accuracy: 0.3000 - val_loss: 0.8662 - val_accuracy: 0.6000
Epoch 2/10
3/3 [==============================] - 3s 916ms/step - loss: 1.2145 - accuracy: 0.3857 - val_loss: 0.9195 - val_accuracy: 0.6000
Epoch 3/10
3/3 [==============================] - 3s 940ms/step - loss: 1.1157 - accuracy: 0.4429 - val_loss: 0.8744 - val_accuracy: 0.7333
Epoch 4/10
3/3 [==============================] - 3s 1s/step - loss: 1.1492 - accuracy: 0.4429 - val_loss: 1.0782 - val_accuracy: 0.4667
Epoch 5/10
3/3 [==============================] - 4s 1s/step - loss: 1.0849 - accuracy: 0.4571 - val_loss: 1.1051 - val_accuracy: 0.4667
Epoch 6/10
3/3 [==============================] - 3s 984ms/step - loss: 1.0274 - accuracy: 0.4286 - val_loss: 1.0260 - val_accuracy: 0.2667
Epoch 6: early stopping

print('CNN5 performance on the test set:')
get_accuracy_metrics(cnn5)

CNN5 performance on the test set:
3/3 [==============================] - 1s 305ms/step
1/1 [==============================] - 0s 268ms/step
1/1 [==============================] - 0s 277ms/step
Train accuracy Score------------>
52.857 %
Val accuracy Score--------->
33.333 %
Test accuracy Score--------->
6.667 %
F1 Score--------------->
9.524 %
Cohen Kappa Score------------->
2.326 %
ROC AUC Score------------->
55.000 %
Recall-------------->
6.667 %
Precision-------------->
16.667 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

CNN6

model = tf.keras.Sequential([
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 32, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Conv2D(filters = 32, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Flatten(),
])

cnn6 = fit_model("CNN6", model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc), training=True)

Model Summary.
Model: "model_12"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_15 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 sequential_13 (Sequential)  (None, 12800)             325004    
                                                                 
 dropout_28 (Dropout)        (None, 12800)             0         
                                                                 
 dense_12 (Dense)            (None, 3)                 38403     
                                                                 
=================================================================
Total params: 363,407
Trainable params: 363,273
Non-trainable params: 134
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 4s 1s/step - loss: 1.7459 - accuracy: 0.3000 - val_loss: 0.7330 - val_accuracy: 0.7333
Epoch 2/10
3/3 [==============================] - 3s 1s/step - loss: 1.1575 - accuracy: 0.4286 - val_loss: 0.9248 - val_accuracy: 0.8667
Epoch 3/10
3/3 [==============================] - 3s 968ms/step - loss: 1.0844 - accuracy: 0.4571 - val_loss: 1.1701 - val_accuracy: 0.2000
Epoch 4/10
3/3 [==============================] - 3s 946ms/step - loss: 1.1349 - accuracy: 0.2714 - val_loss: 0.9220 - val_accuracy: 0.6667
Epoch 5/10
3/3 [==============================] - 3s 928ms/step - loss: 1.1647 - accuracy: 0.4571 - val_loss: 0.9900 - val_accuracy: 0.6000
Epoch 6/10
3/3 [==============================] - 3s 948ms/step - loss: 1.0425 - accuracy: 0.5429 - val_loss: 1.1288 - val_accuracy: 0.2000
Epoch 6: early stopping

print('CNN6 performance on the test set:')
get_accuracy_metrics(cnn6)

CNN6 performance on the test set:
3/3 [==============================] - 1s 318ms/step
1/1 [==============================] - 0s 272ms/step
1/1 [==============================] - 0s 264ms/step
Train accuracy Score------------>
45.714 %
Val accuracy Score--------->
26.667 %
Test accuracy Score--------->
0.000 %
F1 Score--------------->
0.000 %
Cohen Kappa Score------------->
-2.273 %
ROC AUC Score------------->
45.000 %
Recall-------------->
0.000 %
Precision-------------->
0.000 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

CNN7

model = tf.keras.Sequential([
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 32, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Conv2D(filters = 32, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Conv2D(filters = 16, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Flatten(),
])

cnn7 = fit_model("CNN7", model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc), training=True)

Model Summary.
Model: "model_13"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_16 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 sequential_14 (Sequential)  (None, 1600)              329628    
                                                                 
 dropout_32 (Dropout)        (None, 1600)              0         
                                                                 
 dense_13 (Dense)            (None, 3)                 4803      
                                                                 
=================================================================
Total params: 334,431
Trainable params: 334,297
Non-trainable params: 134
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 4s 957ms/step - loss: 1.6848 - accuracy: 0.3143 - val_loss: 1.1505 - val_accuracy: 0.2667
Epoch 2/10
3/3 [==============================] - 3s 937ms/step - loss: 1.1200 - accuracy: 0.4000 - val_loss: 0.8552 - val_accuracy: 0.8000
Epoch 3/10
3/3 [==============================] - 3s 934ms/step - loss: 1.1163 - accuracy: 0.4000 - val_loss: 0.9843 - val_accuracy: 0.5333
Epoch 4/10
3/3 [==============================] - 3s 926ms/step - loss: 1.0382 - accuracy: 0.4286 - val_loss: 1.0534 - val_accuracy: 0.3333
Epoch 5/10
3/3 [==============================] - 3s 938ms/step - loss: 1.0225 - accuracy: 0.4571 - val_loss: 0.9184 - val_accuracy: 0.5333
Epoch 6/10
3/3 [==============================] - 3s 954ms/step - loss: 0.9772 - accuracy: 0.5571 - val_loss: 0.8176 - val_accuracy: 0.6667
Epoch 7/10
3/3 [==============================] - 3s 953ms/step - loss: 0.9926 - accuracy: 0.5000 - val_loss: 1.2017 - val_accuracy: 0.3333
Epoch 8/10
3/3 [==============================] - 3s 933ms/step - loss: 1.0786 - accuracy: 0.4714 - val_loss: 1.2384 - val_accuracy: 0.1333
Epoch 9/10
3/3 [==============================] - 3s 890ms/step - loss: 1.0142 - accuracy: 0.4857 - val_loss: 1.0574 - val_accuracy: 0.4667
Epoch 10/10
3/3 [==============================] - 3s 913ms/step - loss: 0.9705 - accuracy: 0.5857 - val_loss: 0.8623 - val_accuracy: 0.7333

print('CNN7 performance on the test set:')
get_accuracy_metrics(cnn7)

CNN7 performance on the test set:
3/3 [==============================] - 1s 308ms/step
1/1 [==============================] - 0s 265ms/step
1/1 [==============================] - 0s 272ms/step
Train accuracy Score------------>
52.857 %
Val accuracy Score--------->
66.667 %
Test accuracy Score--------->
13.333 %
F1 Score--------------->
8.889 %
Cohen Kappa Score------------->
-11.429 %
ROC AUC Score------------->
30.000 %
Recall-------------->
13.333 %
Precision-------------->
6.667 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

CNN8

model = tf.keras.Sequential([
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 128, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Conv2D(filters = 32, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Conv2D(filters = 32, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.MaxPooling2D(),
    tf.keras.layers.Conv2D(filters = 16, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Conv2D(filters = 16, kernel_size = (3, 3), padding = 'same', activation='relu'),
    tf.keras.layers.Flatten(),
])

cnn8 = fit_model("CNN8", model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc), training=True)

Model Summary.
Model: "model_14"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_17 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 sequential_15 (Sequential)  (None, 1600)              331948    
                                                                 
 dropout_36 (Dropout)        (None, 1600)              0         
                                                                 
 dense_14 (Dense)            (None, 3)                 4803      
                                                                 
=================================================================
Total params: 336,751
Trainable params: 336,617
Non-trainable params: 134
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 4s 1s/step - loss: 1.1980 - accuracy: 0.3143 - val_loss: 0.9724 - val_accuracy: 0.5333
Epoch 2/10
3/3 [==============================] - 3s 967ms/step - loss: 1.0676 - accuracy: 0.4143 - val_loss: 1.0963 - val_accuracy: 0.4000
Epoch 3/10
3/3 [==============================] - 3s 920ms/step - loss: 1.0239 - accuracy: 0.4714 - val_loss: 0.9282 - val_accuracy: 0.4667
Epoch 4/10
3/3 [==============================] - 3s 922ms/step - loss: 1.0743 - accuracy: 0.4714 - val_loss: 0.8561 - val_accuracy: 0.4667
Epoch 5/10
3/3 [==============================] - 3s 929ms/step - loss: 1.0084 - accuracy: 0.4571 - val_loss: 0.9489 - val_accuracy: 0.6000
Epoch 6/10
3/3 [==============================] - 3s 936ms/step - loss: 1.0371 - accuracy: 0.4286 - val_loss: 0.9924 - val_accuracy: 0.3333
Epoch 7/10
3/3 [==============================] - 3s 932ms/step - loss: 0.9820 - accuracy: 0.5571 - val_loss: 0.8944 - val_accuracy: 0.5333
Epoch 8/10
3/3 [==============================] - 3s 918ms/step - loss: 0.9983 - accuracy: 0.4429 - val_loss: 0.7442 - val_accuracy: 0.7333
Epoch 9/10
3/3 [==============================] - 3s 910ms/step - loss: 0.9583 - accuracy: 0.4857 - val_loss: 0.9171 - val_accuracy: 0.6667
Epoch 10/10
3/3 [==============================] - 3s 916ms/step - loss: 0.9977 - accuracy: 0.4714 - val_loss: 1.0547 - val_accuracy: 0.6000

print('CNN8 performance on the test set:')
get_accuracy_metrics(cnn8)

CNN8 performance on the test set:
3/3 [==============================] - 1s 317ms/step
1/1 [==============================] - 0s 258ms/step
1/1 [==============================] - 0s 267ms/step
Train accuracy Score------------>
61.429 %
Val accuracy Score--------->
40.000 %
Test accuracy Score--------->
13.333 %
F1 Score--------------->
13.333 %
Cohen Kappa Score------------->
2.500 %
ROC AUC Score------------->
55.000 %
Recall-------------->
13.333 %
Precision-------------->
13.333 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

ResNet 50

base_model = tf.keras.applications.ResNet50(
    include_top=False,
    weights="imagenet",
    classes=NUM_CLASSES,
)

base_model.trainable = False

resnet50 = fit_model("ResNet50", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))

Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5
94765736/94765736 [==============================] - 11s 0us/step
Model Summary.
Model: "model_15"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_19 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 resnet50 (Functional)       (None, None, None, 2048)  23587712  
                                                                 
 global_average_pooling2d_2   (None, 2048)             0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_37 (Dropout)        (None, 2048)              0         
                                                                 
 dense_15 (Dense)            (None, 3)                 6147      
                                                                 
=================================================================
Total params: 23,593,859
Trainable params: 6,147
Non-trainable params: 23,587,712
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 3s 554ms/step - loss: 1.5936 - accuracy: 0.3857 - val_loss: 1.2789 - val_accuracy: 0.6667
Epoch 2/10
3/3 [==============================] - 1s 329ms/step - loss: 1.5735 - accuracy: 0.5000 - val_loss: 1.4265 - val_accuracy: 0.4667
Epoch 3/10
3/3 [==============================] - 1s 329ms/step - loss: 1.3143 - accuracy: 0.5571 - val_loss: 1.7017 - val_accuracy: 0.3333
Epoch 4/10
3/3 [==============================] - 1s 327ms/step - loss: 1.0376 - accuracy: 0.5143 - val_loss: 1.6307 - val_accuracy: 0.4000
Epoch 5/10
3/3 [==============================] - 1s 324ms/step - loss: 1.0204 - accuracy: 0.5857 - val_loss: 1.4383 - val_accuracy: 0.5333
Epoch 6/10
3/3 [==============================] - 1s 324ms/step - loss: 1.0220 - accuracy: 0.5143 - val_loss: 1.3037 - val_accuracy: 0.5333
Epoch 6: early stopping

print('ResNet50 performance on the test set:')
get_accuracy_metrics(resnet50)

ResNet50 performance on the test set:
3/3 [==============================] - 1s 228ms/step
1/1 [==============================] - 0s 183ms/step
1/1 [==============================] - 0s 178ms/step
Train accuracy Score------------>
57.143 %
Val accuracy Score--------->
53.333 %
Test accuracy Score--------->
33.333 %
F1 Score--------------->
40.000 %
Cohen Kappa Score------------->
-7.143 %
ROC AUC Score------------->
44.000 %
Recall-------------->
33.333 %
Precision-------------->
51.111 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

VGG16

base_model = tf.keras.applications.VGG16(
    include_top=False,
    weights="imagenet",
    input_shape=INPUT_SHAPE,
    classes=NUM_CLASSES,
    classifier_activation="softmax",
)

base_model.trainable = False

Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/vgg16/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5
58889256/58889256 [==============================] - 7s 0us/step

vgg16 = fit_model("VGG16", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))

Model Summary.
Model: "model_16"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_21 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 vgg16 (Functional)          (None, 5, 5, 512)         14714688  
                                                                 
 global_average_pooling2d_3   (None, 512)              0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_38 (Dropout)        (None, 512)               0         
                                                                 
 dense_16 (Dense)            (None, 3)                 1539      
                                                                 
=================================================================
Total params: 14,716,227
Trainable params: 1,539
Non-trainable params: 14,714,688
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 3s 957ms/step - loss: 5.2476 - accuracy: 0.3857 - val_loss: 4.4457 - val_accuracy: 0.4667
Epoch 2/10
3/3 [==============================] - 3s 905ms/step - loss: 4.2855 - accuracy: 0.4429 - val_loss: 4.3819 - val_accuracy: 0.4667
Epoch 3/10
3/3 [==============================] - 3s 890ms/step - loss: 4.3707 - accuracy: 0.4571 - val_loss: 4.3926 - val_accuracy: 0.4667
Epoch 4/10
3/3 [==============================] - 3s 920ms/step - loss: 4.1395 - accuracy: 0.4714 - val_loss: 4.5551 - val_accuracy: 0.4667
Epoch 5/10
3/3 [==============================] - 3s 886ms/step - loss: 2.9570 - accuracy: 0.5429 - val_loss: 4.7753 - val_accuracy: 0.4667
Epoch 6/10
3/3 [==============================] - 3s 897ms/step - loss: 4.1382 - accuracy: 0.5286 - val_loss: 5.0092 - val_accuracy: 0.4000
Epoch 7/10
3/3 [==============================] - 3s 916ms/step - loss: 3.6442 - accuracy: 0.4857 - val_loss: 5.0746 - val_accuracy: 0.4000
Epoch 7: early stopping

print('VGG16 performance on the test set:')
get_accuracy_metrics(vgg16)

VGG16 performance on the test set:
3/3 [==============================] - 2s 623ms/step
1/1 [==============================] - 1s 518ms/step
1/1 [==============================] - 1s 511ms/step
Train accuracy Score------------>
50.000 %
Val accuracy Score--------->
40.000 %
Test accuracy Score--------->
13.333 %
F1 Score--------------->
20.513 %
Cohen Kappa Score------------->
-2.632 %
ROC AUC Score------------->
46.000 %
Recall-------------->
13.333 %
Precision-------------->
44.444 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

VGG19

base_model = tf.keras.applications.VGG19(
    include_top=False,
    weights="imagenet",
    input_shape=INPUT_SHAPE,
    classes=NUM_CLASSES,
    classifier_activation="softmax",
)

base_model.trainable = False

Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/vgg19/vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5
80134624/80134624 [==============================] - 8s 0us/step

vgg19 = fit_model("VGG19", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))

Model Summary.
Model: "model_17"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_23 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 vgg19 (Functional)          (None, 5, 5, 512)         20024384  
                                                                 
 global_average_pooling2d_4   (None, 512)              0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_39 (Dropout)        (None, 512)               0         
                                                                 
 dense_17 (Dense)            (None, 3)                 1539      
                                                                 
=================================================================
Total params: 20,025,923
Trainable params: 1,539
Non-trainable params: 20,024,384
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 4s 1s/step - loss: 9.2581 - accuracy: 0.3000 - val_loss: 8.9689 - val_accuracy: 0.2000
Epoch 2/10
3/3 [==============================] - 3s 1s/step - loss: 6.6118 - accuracy: 0.3429 - val_loss: 6.0221 - val_accuracy: 0.3333
Epoch 3/10
3/3 [==============================] - 3s 1s/step - loss: 7.3799 - accuracy: 0.2714 - val_loss: 4.4366 - val_accuracy: 0.4000
Epoch 4/10
3/3 [==============================] - 3s 1s/step - loss: 6.4170 - accuracy: 0.3429 - val_loss: 3.6450 - val_accuracy: 0.4667
Epoch 5/10
3/3 [==============================] - 3s 1s/step - loss: 5.6530 - accuracy: 0.3143 - val_loss: 3.3127 - val_accuracy: 0.5333
Epoch 6/10
3/3 [==============================] - 3s 1s/step - loss: 5.5191 - accuracy: 0.3000 - val_loss: 2.9773 - val_accuracy: 0.4667
Epoch 7/10
3/3 [==============================] - 3s 1s/step - loss: 5.1576 - accuracy: 0.3857 - val_loss: 2.7574 - val_accuracy: 0.4000
Epoch 8/10
3/3 [==============================] - 3s 1s/step - loss: 4.2287 - accuracy: 0.4143 - val_loss: 2.8127 - val_accuracy: 0.4000
Epoch 9/10
3/3 [==============================] - 4s 1s/step - loss: 3.9558 - accuracy: 0.4714 - val_loss: 2.7563 - val_accuracy: 0.3333
Epoch 10/10
3/3 [==============================] - 3s 1s/step - loss: 4.3099 - accuracy: 0.3857 - val_loss: 2.8368 - val_accuracy: 0.3333

print('VGG19 performance on the test set:')
get_accuracy_metrics(vgg19)

VGG19 performance on the test set:
3/3 [==============================] - 3s 791ms/step
1/1 [==============================] - 1s 608ms/step
1/1 [==============================] - 1s 636ms/step
Train accuracy Score------------>
45.714 %
Val accuracy Score--------->
33.333 %
Test accuracy Score--------->
26.667 %
F1 Score--------------->
35.556 %
Cohen Kappa Score------------->
8.333 %
ROC AUC Score------------->
41.000 %
Recall-------------->
26.667 %
Precision-------------->
80.000 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

MobileNet Pretranined

base_model = tf.keras.applications.MobileNet(include_top=False, 
                                               weights='imagenet', 
                                               input_shape=INPUT_SHAPE)

base_model.trainable = False

Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/mobilenet/mobilenet_1_0_160_tf_no_top.h5
17225924/17225924 [==============================] - 2s 0us/step

mobilenet = fit_model("MobileNet", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))

Model Summary.
Model: "model_18"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_25 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 mobilenet_1.00_160 (Functio  (None, 5, 5, 1024)       3228864   
 nal)                                                            
                                                                 
 global_average_pooling2d_5   (None, 1024)             0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_40 (Dropout)        (None, 1024)              0         
                                                                 
 dense_18 (Dense)            (None, 3)                 3075      
                                                                 
=================================================================
Total params: 3,231,939
Trainable params: 3,075
Non-trainable params: 3,228,864
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 1s 209ms/step - loss: 1.3250 - accuracy: 0.4429 - val_loss: 1.2073 - val_accuracy: 0.4000
Epoch 2/10
3/3 [==============================] - 0s 86ms/step - loss: 1.1244 - accuracy: 0.4286 - val_loss: 1.5010 - val_accuracy: 0.2000
Epoch 3/10
3/3 [==============================] - 0s 82ms/step - loss: 1.2593 - accuracy: 0.4286 - val_loss: 1.3026 - val_accuracy: 0.4667
Epoch 4/10
3/3 [==============================] - 0s 81ms/step - loss: 0.9933 - accuracy: 0.4857 - val_loss: 1.1075 - val_accuracy: 0.4667
Epoch 5/10
3/3 [==============================] - 0s 85ms/step - loss: 1.0599 - accuracy: 0.5000 - val_loss: 1.0483 - val_accuracy: 0.5333
Epoch 6/10
3/3 [==============================] - 0s 87ms/step - loss: 1.1146 - accuracy: 0.4429 - val_loss: 1.0646 - val_accuracy: 0.5333
Epoch 7/10
3/3 [==============================] - 0s 85ms/step - loss: 0.9108 - accuracy: 0.5571 - val_loss: 1.0995 - val_accuracy: 0.4667
Epoch 8/10
3/3 [==============================] - 0s 87ms/step - loss: 0.9597 - accuracy: 0.5286 - val_loss: 1.0835 - val_accuracy: 0.4667
Epoch 9/10
3/3 [==============================] - 0s 85ms/step - loss: 1.0522 - accuracy: 0.5000 - val_loss: 1.0310 - val_accuracy: 0.5333
Epoch 10/10
3/3 [==============================] - 0s 83ms/step - loss: 0.9976 - accuracy: 0.6000 - val_loss: 0.9536 - val_accuracy: 0.4667

print('MobileNet performance on the test set:')
get_accuracy_metrics(mobilenet)

MobileNet performance on the test set:
3/3 [==============================] - 0s 51ms/step
1/1 [==============================] - 0s 49ms/step
1/1 [==============================] - 0s 51ms/step
Train accuracy Score------------>
55.714 %
Val accuracy Score--------->
46.667 %
Test accuracy Score--------->
26.667 %
F1 Score--------------->
37.436 %
Cohen Kappa Score------------->
2.941 %
ROC AUC Score------------->
22.000 %
Recall-------------->
26.667 %
Precision-------------->
73.333 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

MobileNet V2 pre-trained

base_model = tf.keras.applications.MobileNetV2(include_top=False, 
                                               weights='imagenet', 
                                               input_shape=INPUT_SHAPE)

base_model.trainable = False

Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/mobilenet_v2/mobilenet_v2_weights_tf_dim_ordering_tf_kernels_1.0_160_no_top.h5
9406464/9406464 [==============================] - 1s 0us/step

mobilenetv2 = fit_model("MobileNetV2", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))

Model Summary.
Model: "model_19"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_27 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 mobilenetv2_1.00_160 (Funct  (None, 5, 5, 1280)       2257984   
 ional)                                                          
                                                                 
 global_average_pooling2d_6   (None, 1280)             0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_41 (Dropout)        (None, 1280)              0         
                                                                 
 dense_19 (Dense)            (None, 3)                 3843      
                                                                 
=================================================================
Total params: 2,261,827
Trainable params: 3,843
Non-trainable params: 2,257,984
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 2s 304ms/step - loss: 1.8470 - accuracy: 0.2286 - val_loss: 1.1985 - val_accuracy: 0.4667
Epoch 2/10
3/3 [==============================] - 0s 82ms/step - loss: 1.4548 - accuracy: 0.3143 - val_loss: 0.8727 - val_accuracy: 0.6000
Epoch 3/10
3/3 [==============================] - 0s 85ms/step - loss: 1.3060 - accuracy: 0.4143 - val_loss: 1.0550 - val_accuracy: 0.4000
Epoch 4/10
3/3 [==============================] - 0s 82ms/step - loss: 1.1841 - accuracy: 0.4714 - val_loss: 1.4036 - val_accuracy: 0.4000
Epoch 5/10
3/3 [==============================] - 0s 89ms/step - loss: 1.1979 - accuracy: 0.4143 - val_loss: 1.3764 - val_accuracy: 0.3333
Epoch 6/10
3/3 [==============================] - 0s 79ms/step - loss: 1.2333 - accuracy: 0.4143 - val_loss: 1.1880 - val_accuracy: 0.4000
Epoch 7/10
3/3 [==============================] - 0s 81ms/step - loss: 1.1137 - accuracy: 0.4714 - val_loss: 1.1701 - val_accuracy: 0.4000
Epoch 7: early stopping

print('MobileNetV2 performance on the test set:')
get_accuracy_metrics(mobilenetv2)

MobileNetV2 performance on the test set:
3/3 [==============================] - 1s 53ms/step
1/1 [==============================] - 0s 47ms/step
1/1 [==============================] - 0s 45ms/step
Train accuracy Score------------>
50.000 %
Val accuracy Score--------->
40.000 %
Test accuracy Score--------->
6.667 %
F1 Score--------------->
6.061 %
Cohen Kappa Score------------->
-13.514 %
ROC AUC Score------------->
50.000 %
Recall-------------->
6.667 %
Precision-------------->
5.556 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

Inception V3

base_model = tf.keras.applications.InceptionV3(
    include_top=False,
    weights="imagenet",
    input_shape=INPUT_SHAPE,
    classes=NUM_CLASSES,
    classifier_activation="softmax",
)

base_model.trainable = False

incpetionv3 = fit_model("InceptionV3", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))

Model Summary.
Model: "model_20"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_29 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 inception_v3 (Functional)   (None, 3, 3, 2048)        21802784  
                                                                 
 global_average_pooling2d_7   (None, 2048)             0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_42 (Dropout)        (None, 2048)              0         
                                                                 
 dense_20 (Dense)            (None, 3)                 6147      
                                                                 
=================================================================
Total params: 21,808,931
Trainable params: 6,147
Non-trainable params: 21,802,784
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 3s 443ms/step - loss: 50.7983 - accuracy: 0.2857 - val_loss: 6.4663 - val_accuracy: 0.6000
Epoch 2/10
3/3 [==============================] - 0s 151ms/step - loss: 27.2885 - accuracy: 0.4714 - val_loss: 9.8325 - val_accuracy: 0.8000
Epoch 3/10
3/3 [==============================] - 0s 144ms/step - loss: 30.6071 - accuracy: 0.4714 - val_loss: 7.0202 - val_accuracy: 0.6000
Epoch 4/10
3/3 [==============================] - 0s 143ms/step - loss: 21.1291 - accuracy: 0.5000 - val_loss: 6.0973 - val_accuracy: 0.6000
Epoch 5/10
3/3 [==============================] - 0s 144ms/step - loss: 19.5554 - accuracy: 0.3714 - val_loss: 14.1759 - val_accuracy: 0.4000
Epoch 6/10
3/3 [==============================] - 0s 142ms/step - loss: 25.3541 - accuracy: 0.3286 - val_loss: 3.9975 - val_accuracy: 0.6000
Epoch 7/10
3/3 [==============================] - 0s 144ms/step - loss: 15.8824 - accuracy: 0.5143 - val_loss: 1.7173 - val_accuracy: 0.8667
Epoch 8/10
3/3 [==============================] - 0s 144ms/step - loss: 21.8860 - accuracy: 0.4286 - val_loss: 1.7265 - val_accuracy: 0.7333
Epoch 9/10
3/3 [==============================] - 0s 145ms/step - loss: 17.9414 - accuracy: 0.5286 - val_loss: 1.0400 - val_accuracy: 0.7333
Epoch 10/10
3/3 [==============================] - 0s 142ms/step - loss: 19.0953 - accuracy: 0.4286 - val_loss: 1.6771 - val_accuracy: 0.8000

print('InceptionV3 performance on the test set:')
get_accuracy_metrics(incpetionv3)

InceptionV3 performance on the test set:
3/3 [==============================] - 1s 93ms/step
1/1 [==============================] - 0s 83ms/step
1/1 [==============================] - 0s 83ms/step
Train accuracy Score------------>
61.429 %
Val accuracy Score--------->
80.000 %
Test accuracy Score--------->
26.667 %
F1 Score--------------->
25.455 %
Cohen Kappa Score------------->
0.000 %
ROC AUC Score------------->
37.000 %
Recall-------------->
26.667 %
Precision-------------->
76.667 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

DenseNet 121

base_model = tf.keras.applications.DenseNet121(
    include_top=False,
    weights="imagenet",
    input_shape=INPUT_SHAPE,
    classes=NUM_CLASSES,
    classifier_activation="softmax",
)

base_model.trainable = False

densenet121 = fit_model("DenseNet121", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))

Model Summary.
Model: "model_21"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_31 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 densenet121 (Functional)    (None, 5, 5, 1024)        7037504   
                                                                 
 global_average_pooling2d_8   (None, 1024)             0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_43 (Dropout)        (None, 1024)              0         
                                                                 
 dense_21 (Dense)            (None, 3)                 3075      
                                                                 
=================================================================
Total params: 7,040,579
Trainable params: 3,075
Non-trainable params: 7,037,504
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 5s 688ms/step - loss: 8.9645 - accuracy: 0.2571 - val_loss: 3.7331 - val_accuracy: 0.1333
Epoch 2/10
3/3 [==============================] - 1s 296ms/step - loss: 5.6826 - accuracy: 0.3571 - val_loss: 3.8929 - val_accuracy: 0.1333
Epoch 3/10
3/3 [==============================] - 1s 282ms/step - loss: 5.4314 - accuracy: 0.4143 - val_loss: 2.9998 - val_accuracy: 0.3333
Epoch 4/10
3/3 [==============================] - 1s 283ms/step - loss: 5.0640 - accuracy: 0.4857 - val_loss: 2.8958 - val_accuracy: 0.3333
Epoch 5/10
3/3 [==============================] - 1s 278ms/step - loss: 4.2808 - accuracy: 0.5143 - val_loss: 3.0649 - val_accuracy: 0.3333
Epoch 6/10
3/3 [==============================] - 1s 283ms/step - loss: 5.0579 - accuracy: 0.4714 - val_loss: 3.3136 - val_accuracy: 0.2667
Epoch 7/10
3/3 [==============================] - 1s 286ms/step - loss: 3.3413 - accuracy: 0.5429 - val_loss: 4.1449 - val_accuracy: 0.2667
Epoch 8/10
3/3 [==============================] - 1s 278ms/step - loss: 4.7983 - accuracy: 0.4571 - val_loss: 4.1632 - val_accuracy: 0.2667
Epoch 9/10
3/3 [==============================] - 1s 277ms/step - loss: 3.8919 - accuracy: 0.4571 - val_loss: 3.7137 - val_accuracy: 0.2667
Epoch 9: early stopping

print('DenseNet121 performance on the test set:')
get_accuracy_metrics(densenet121)

DenseNet121 performance on the test set:
3/3 [==============================] - 1s 191ms/step
1/1 [==============================] - 0s 158ms/step
1/1 [==============================] - 0s 163ms/step
Train accuracy Score------------>
55.714 %
Val accuracy Score--------->
26.667 %
Test accuracy Score--------->
26.667 %
F1 Score--------------->
38.889 %
Cohen Kappa Score------------->
13.158 %
ROC AUC Score------------->
51.000 %
Recall-------------->
26.667 %
Precision-------------->
88.889 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

DenseNet 169

base_model = tf.keras.applications.DenseNet169(
    include_top=False,
    weights="imagenet",
    input_shape=INPUT_SHAPE,
    classes=NUM_CLASSES,
    classifier_activation="softmax",
)

base_model.trainable = False

densenet169 = fit_model("DenseNet169", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))

Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/densenet/densenet169_weights_tf_dim_ordering_tf_kernels_notop.h5
51877672/51877672 [==============================] - 5s 0us/step
Model Summary.
Model: "model_22"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_33 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 densenet169 (Functional)    (None, 5, 5, 1664)        12642880  
                                                                 
 global_average_pooling2d_9   (None, 1664)             0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dropout_44 (Dropout)        (None, 1664)              0         
                                                                 
 dense_22 (Dense)            (None, 3)                 4995      
                                                                 
=================================================================
Total params: 12,647,875
Trainable params: 4,995
Non-trainable params: 12,642,880
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 6s 934ms/step - loss: 12.7267 - accuracy: 0.2714 - val_loss: 4.5255 - val_accuracy: 0.4667
Epoch 2/10
3/3 [==============================] - 1s 340ms/step - loss: 8.7353 - accuracy: 0.4571 - val_loss: 3.0815 - val_accuracy: 0.6667
Epoch 3/10
3/3 [==============================] - 1s 340ms/step - loss: 11.5507 - accuracy: 0.3429 - val_loss: 2.8783 - val_accuracy: 0.6667
Epoch 4/10
3/3 [==============================] - 1s 344ms/step - loss: 7.1117 - accuracy: 0.4714 - val_loss: 3.5252 - val_accuracy: 0.4000
Epoch 5/10
3/3 [==============================] - 1s 337ms/step - loss: 6.5601 - accuracy: 0.4000 - val_loss: 7.6735 - val_accuracy: 0.2000
Epoch 6/10
3/3 [==============================] - 1s 343ms/step - loss: 6.8822 - accuracy: 0.3714 - val_loss: 7.9800 - val_accuracy: 0.1333
Epoch 7/10
3/3 [==============================] - 1s 336ms/step - loss: 6.1544 - accuracy: 0.3857 - val_loss: 5.3288 - val_accuracy: 0.3333
Epoch 8/10
3/3 [==============================] - 1s 333ms/step - loss: 5.3028 - accuracy: 0.4571 - val_loss: 3.8081 - val_accuracy: 0.4000
Epoch 8: early stopping

print('DenseNet169 performance on the test set:')
get_accuracy_metrics(densenet169)

DenseNet169 performance on the test set:
3/3 [==============================] - 2s 233ms/step
1/1 [==============================] - 0s 193ms/step
1/1 [==============================] - 0s 187ms/step
Train accuracy Score------------>
47.143 %
Val accuracy Score--------->
40.000 %
Test accuracy Score--------->
20.000 %
F1 Score--------------->
24.242 %
Cohen Kappa Score------------->
2.703 %
ROC AUC Score------------->
43.000 %
Recall-------------->
20.000 %
Precision-------------->
77.778 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

InceptionResNetv2

base_model = tf.keras.applications.InceptionResNetV2(
    include_top=False,
    weights="imagenet",
    input_shape=INPUT_SHAPE,
    classes=NUM_CLASSES,
    classifier_activation="softmax",
)

base_model.trainable = False

inceptionresnetv2 = fit_model("InceptionResNetV2", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))

Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/inception_resnet_v2/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5
219055592/219055592 [==============================] - 26s 0us/step
Model Summary.
Model: "model_23"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_35 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 inception_resnet_v2 (Functi  (None, 3, 3, 1536)       54336736  
 onal)                                                           
                                                                 
 global_average_pooling2d_10  (None, 1536)             0         
  (GlobalAveragePooling2D)                                       
                                                                 
 dropout_45 (Dropout)        (None, 1536)              0         
                                                                 
 dense_23 (Dense)            (None, 3)                 4611      
                                                                 
=================================================================
Total params: 54,341,347
Trainable params: 4,611
Non-trainable params: 54,336,736
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 7s 1s/step - loss: 153.0160 - accuracy: 0.2857 - val_loss: 102.0554 - val_accuracy: 0.1333
Epoch 2/10
3/3 [==============================] - 1s 356ms/step - loss: 154.2074 - accuracy: 0.4714 - val_loss: 47.5271 - val_accuracy: 0.8000
Epoch 3/10
3/3 [==============================] - 1s 365ms/step - loss: 144.2635 - accuracy: 0.4571 - val_loss: 89.0682 - val_accuracy: 0.1333
Epoch 4/10
3/3 [==============================] - 1s 351ms/step - loss: 137.2067 - accuracy: 0.4143 - val_loss: 38.8472 - val_accuracy: 0.2000
Epoch 5/10
3/3 [==============================] - 1s 359ms/step - loss: 90.6458 - accuracy: 0.4286 - val_loss: 32.4482 - val_accuracy: 0.8000
Epoch 6/10
3/3 [==============================] - 1s 355ms/step - loss: 128.8864 - accuracy: 0.3429 - val_loss: 35.8594 - val_accuracy: 0.8000
Epoch 7/10
3/3 [==============================] - 1s 353ms/step - loss: 102.2348 - accuracy: 0.3714 - val_loss: 14.2647 - val_accuracy: 0.7333
Epoch 8/10
3/3 [==============================] - 1s 358ms/step - loss: 106.2678 - accuracy: 0.3857 - val_loss: 120.1697 - val_accuracy: 0.1333
Epoch 9/10
3/3 [==============================] - 1s 359ms/step - loss: 153.3878 - accuracy: 0.3571 - val_loss: 20.0372 - val_accuracy: 0.1333
Epoch 10/10
3/3 [==============================] - 1s 358ms/step - loss: 104.0266 - accuracy: 0.4000 - val_loss: 31.8369 - val_accuracy: 0.8000

print('InceptionResNetV2 performance on the test set:')
get_accuracy_metrics(inceptionresnetv2)

InceptionResNetV2 performance on the test set:
3/3 [==============================] - 2s 240ms/step
1/1 [==============================] - 0s 198ms/step
1/1 [==============================] - 0s 206ms/step
Train accuracy Score------------>
42.857 %
Val accuracy Score--------->
80.000 %
Test accuracy Score--------->
33.333 %
F1 Score--------------->
16.667 %
Cohen Kappa Score------------->
0.000 %
ROC AUC Score------------->
50.000 %
Recall-------------->
33.333 %
Precision-------------->
11.111 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

ResNet101

base_model = tf.keras.applications.ResNet101(
    include_top=False,
    weights="imagenet",
    input_shape=INPUT_SHAPE,
    classes=NUM_CLASSES,
    classifier_activation="softmax",
)

base_model.trainable = False

resnet101 = fit_model("ResNet101", base_model, train_features, train_labels_1hotenc, (val_features, val_labels_1hotenc))

Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet101_weights_tf_dim_ordering_tf_kernels_notop.h5
171446536/171446536 [==============================] - 19s 0us/step
Model Summary.
Model: "model_24"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_37 (InputLayer)       [(None, 160, 160, 3)]     0         
                                                                 
 sequential_8 (Sequential)   (None, 160, 160, 3)       0         
                                                                 
 resnet101 (Functional)      (None, 5, 5, 2048)        42658176  
                                                                 
 global_average_pooling2d_11  (None, 2048)             0         
  (GlobalAveragePooling2D)                                       
                                                                 
 dropout_46 (Dropout)        (None, 2048)              0         
                                                                 
 dense_24 (Dense)            (None, 3)                 6147      
                                                                 
=================================================================
Total params: 42,664,323
Trainable params: 6,147
Non-trainable params: 42,658,176
_________________________________________________________________
None
Epoch 1/10
3/3 [==============================] - 6s 1s/step - loss: 1.4095 - accuracy: 0.4571 - val_loss: 1.0700 - val_accuracy: 0.6000
Epoch 2/10
3/3 [==============================] - 2s 574ms/step - loss: 1.0785 - accuracy: 0.5714 - val_loss: 1.1273 - val_accuracy: 0.5333
Epoch 3/10
3/3 [==============================] - 2s 576ms/step - loss: 0.9201 - accuracy: 0.6286 - val_loss: 1.6148 - val_accuracy: 0.4000
Epoch 4/10
3/3 [==============================] - 2s 578ms/step - loss: 0.9295 - accuracy: 0.6143 - val_loss: 1.6263 - val_accuracy: 0.4000
Epoch 5/10
3/3 [==============================] - 2s 577ms/step - loss: 1.0989 - accuracy: 0.6429 - val_loss: 1.3021 - val_accuracy: 0.4667
Epoch 6/10
3/3 [==============================] - 2s 580ms/step - loss: 0.9922 - accuracy: 0.5857 - val_loss: 1.2924 - val_accuracy: 0.4000
Epoch 6: early stopping

print('ResNet101 performance on the test set:')
get_accuracy_metrics(resnet101)

ResNet101 performance on the test set:
3/3 [==============================] - 2s 411ms/step
1/1 [==============================] - 0s 321ms/step
1/1 [==============================] - 0s 322ms/step
Train accuracy Score------------>
65.714 %
Val accuracy Score--------->
40.000 %
Test accuracy Score--------->
13.333 %
F1 Score--------------->
15.812 %
Cohen Kappa Score------------->
-21.875 %
ROC AUC Score------------->
56.000 %
Recall-------------->
13.333 %
Precision-------------->
26.984 %

/home/usairim/.local/share/virtualenvs/Project-KXpvUfAX/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))